package 搜索页条目;

import Moudle.OutItem;
import MyException.VerificationCodeAppearsException;
import Utils.HttpConnect;
import Utils.Tools;
import com.mongodb.MongoClient;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import MyException.InvalidCookieException;

import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class Main {



    static final String START_DATE = "2011-05-31";
    static final String END_DATE = "2021-07-31";

    static final int REQUEST_DELAY = 0;

    static  MongoDatabase mDatabase =  new MongoClient("192.168.1.173", 27017)
            .getDatabase("中国知网-博硕士");
    static  MongoCollection SELECT_PAGE_COLLECTION = mDatabase.getCollection("SelectPageItem");
    static  MongoCollection HAVE_DATA_DATE_COLLECTION = mDatabase.getCollection("HaveDataDate");


    static Map<String, String> headersMap = new HashMap<>();
    static {
//        headersMap.put("Cookie", Tools.getCookie());
        headersMap.put("Cookie", "cnkiUserKey=a8d80371-908c-f226-db63-ab7e82d76c05; Ecp_ClientId=8210817132501221482; Ecp_IpLoginFail=210817116.234.209.223; RsPerPage=20; SID_epub=epub120105; Ecp_ClientIp=116.234.209.223; _pk_ses=*; ASP.NET_SessionId=owqibmfo2nyncr2kbngnpgjp; _pk_id=807555a1-c1f3-4fa1-a425-079376cfb879.1629177960.2.1629189776.1629187814.");
        headersMap.put("Host", "epub.cnki.net");
        headersMap.put("Referer", "https://epub.cnki.net/kns/brief/result.aspx?dbPrefix=CDMD");
    }

    public static void main(String[] args) throws InvalidCookieException, VerificationCodeAppearsException, IOException, InterruptedException {

        List<String> dateList = Tools.getDateList(START_DATE, END_DATE);
        for (String date : dateList) {
            Tools.consoleInfo("开始抓取：【" + date + "】的数据！");
            /**
             * 指定一天的数据
             * 知网根据cookie来返回抓取的日期
             */
            String url = "https://epub.cnki.net/kns/request/SearchHandler.ashx?DbCatalog=%e4%b8%ad%e5%9b%bd%e4%bc%98%e7%a7%80%e5%8d%9a%e7%a1%95%e5%a3%ab%e5%ad%a6%e4%bd%8d%e8%ae%ba%e6%96%87%e5%85%a8%e6%96%87%e6%95%b0%e6%8d%ae%e5%ba%93&ConfigFile=CDMD.xml&db_opt=%E4%B8%AD%E5%9B%BD%E4%BC%98%E7%A7%80%E5%8D%9A%E7%A1%95%E5%A3%AB%E5%AD%A6%E4%BD%8D%E8%AE%BA%E6%96%87%E5%85%A8%E6%96%87%E6%95%B0%E6%8D%AE%E5%BA%93&db_value=%E4%B8%AD%E5%9B%BD%E5%8D%9A%E5%A3%AB%E5%AD%A6%E4%BD%8D%E8%AE%BA%E6%96%87%E5%85%A8%E6%96%87%E6%95%B0%E6%8D%AE%E5%BA%93%2C%E4%B8%AD%E5%9B%BD%E4%BC%98%E7%A7%80%E7%A1%95%E5%A3%AB%E5%AD%A6%E4%BD%8D%E8%AE%BA%E6%96%87%E5%85%A8%E6%96%87%E6%95%B0%E6%8D%AE%E5%BA%93&base_special1=%25&degreedanwei_special1=%25&updatedateN_to=" + date + "&PageName=ASP.brief_result_aspx&DbPrefix=CDMD&updatedateN_from=" + date;
            String responseStr = HttpGet(url, headersMap, date, 0);
            //System.out.println(responseStr);

            /**
             * 固定url抓取这一天第一页的数据
             */
            url = "https://epub.cnki.net/kns/brief/brief.aspx?pagename="+responseStr;
            responseStr = HttpGet(url, headersMap, date, 1);

            /**
             * 检查这一天是否有数据
             */
            if (!hasData(responseStr, date)){
                Tools.consoleInfo("【" + date + "】没有数据！");
                continue;
            }

            /**
             * 记录这一天 和 这一天的条目数量
             */
            recordDate(responseStr, date);

            /**
             * 解析html 获取当天所有数据
             */
            getHtml(url, responseStr, date, 1);

            System.out.println(date);
        }
    }

    public static void recordDate(String html, String date){
        /**
         * 记录这一天 和 其拥有的条目数量
         */
        org.bson.Document document = new org.bson.Document("_id", date);
        String dataInfo = Jsoup.parse(html).getElementsByClass("pagerTitleCell").get(0).text();
        document.append("quanity", dataInfo.replaceAll("找到|条结果|,| ", ""));
        Tools.updateToDb(HAVE_DATA_DATE_COLLECTION, document);
    }

    public static boolean hasData(String html, String date) throws InvalidCookieException, VerificationCodeAppearsException {
        /**
         * 解析html
         * 判断这一天是否有数据
         */
        String dataInfo = Jsoup.parse(html).getElementsByClass("pagerTitleCell").get(0).text();
        System.out.println("【" + date + "】" + dataInfo);
        return !dataInfo.contains("找到 0 条结果");
    }

    public static void getHtml(String url, String html, String dateInfo, int pageIndex) throws InvalidCookieException, VerificationCodeAppearsException, IOException, InterruptedException {
        /**
         * 解析html 获取item
         * item写入数据库
         * 判断是否还有下一页 若有 递归
         */
        Tools.consoleInfo("正在解析 【" + dateInfo + "】, 第"+ pageIndex+"页...");
        Document htmlDoc = Jsoup.parse(html);
        parseHtml(htmlDoc);
//        try {
//            parseHtml(htmlDoc);
//        }catch (Exception e){
//            getHtml( url, HttpConnect.HttpGet_OkHttp(url, headersMap, dateInfo, pageIndex), dateInfo, pageIndex);
//            return;
//        }


        Element lastTagA = htmlDoc.getElementsByClass("TitleLeftCell").first().getElementsByTag("a").last();
        if (lastTagA == null || !lastTagA.text().equals("下一页")) {
            return;
        }

        url = "https://epub.cnki.net/kns/brief/brief.aspx" + lastTagA.attr("href");
        getHtml( url, HttpGet(url, headersMap, dateInfo, pageIndex), dateInfo, pageIndex+1);
    }


    public static void parseHtml(Document htmlDoc){
        /**
         * 解析html
         * 提取item
         */
        Elements tag_trs = htmlDoc.getElementsByClass("GridTableContent").first().getElementsByAttribute("bgcolor");
        int tagLength = tag_trs.size();

        for (int i = 1; i < tagLength; i++) {
            Elements tag_tds = tag_trs.get(i).getElementsByTag("td");

            OutItem item = new OutItem();
            item.url = tag_tds.get(1).child(0).attr("href");
            item.select_title = item.get_select_title(tag_tds.get(1).getElementsByTag("script").first().childNodes().get(0).attr("data"));
            item.select_author = tag_tds.get(2).text();
            item.select_学位授予单位 = tag_tds.get(3).text();
            item.select_degree = tag_tds.get(4).text();
            item.select_get_degree_year = tag_tds.get(5).text();
            item.select_quote_count = tag_tds.get(6).text();
            try {
                item.select_quote_url = tag_tds.get(6).getElementsByTag("a").first().attr("href");
            }catch (NullPointerException e){
                item.select_quote_url = "";
            }

            item.select_download_count = tag_tds.get(7).text();

            Tools.updateToDb(SELECT_PAGE_COLLECTION, item.toDocument());
        }

    }

    public static String HttpGet(String url, Map<String, String> header_map, String date, int curPage) throws IOException, InterruptedException {
        String responseStr = HttpConnect.HttpGet_OkHttp(url, headersMap);
        try {
            HttpConnect.checkHtml(responseStr, url);
            return responseStr;
        }catch (InvalidCookieException | VerificationCodeAppearsException e){
            if (e instanceof InvalidCookieException)
                System.out.println("Cookie = " + headersMap.get("Cookie"));
            e.printStackTrace();
            url = getNewUrl(date, curPage);
            return HttpGet(url, headersMap, date, curPage);
        }

    }

    public static String getNewUrl(String date, int curPage) throws IOException {
        int changeCookieTime = 0;
        while (changeCookieTime <= 20){
            headersMap.put("Cookie", Tools.getCookie());
            try {
                String html = isValidCookie(date);
                return getNextUrl(html, curPage);
            } catch (InvalidCookieException | VerificationCodeAppearsException e) {
                e.printStackTrace();
                changeCookieTime++;
                //Tools.consoleInfo("休眠10秒后重试！");
            }
        }
        throw new IOException("无法获取新的cookie！");
    }

    public static String isValidCookie(String date) throws IOException, InvalidCookieException, VerificationCodeAppearsException {
        String url = "https://epub.cnki.net/kns/request/SearchHandler.ashx?DbCatalog=%e4%b8%ad%e5%9b%bd%e4%bc%98%e7%a7%80%e5%8d%9a%e7%a1%95%e5%a3%ab%e5%ad%a6%e4%bd%8d%e8%ae%ba%e6%96%87%e5%85%a8%e6%96%87%e6%95%b0%e6%8d%ae%e5%ba%93&ConfigFile=CDMD.xml&db_opt=%E4%B8%AD%E5%9B%BD%E4%BC%98%E7%A7%80%E5%8D%9A%E7%A1%95%E5%A3%AB%E5%AD%A6%E4%BD%8D%E8%AE%BA%E6%96%87%E5%85%A8%E6%96%87%E6%95%B0%E6%8D%AE%E5%BA%93&db_value=%E4%B8%AD%E5%9B%BD%E5%8D%9A%E5%A3%AB%E5%AD%A6%E4%BD%8D%E8%AE%BA%E6%96%87%E5%85%A8%E6%96%87%E6%95%B0%E6%8D%AE%E5%BA%93%2C%E4%B8%AD%E5%9B%BD%E4%BC%98%E7%A7%80%E7%A1%95%E5%A3%AB%E5%AD%A6%E4%BD%8D%E8%AE%BA%E6%96%87%E5%85%A8%E6%96%87%E6%95%B0%E6%8D%AE%E5%BA%93&base_special1=%25&degreedanwei_special1=%25&updatedateN_to=" + date + "&PageName=ASP.brief_result_aspx&DbPrefix=CDMD&updatedateN_from=" + date;
        String responseStr = HttpConnect.HttpGet_OkHttp(url, headersMap);
        //System.out.println(responseStr);

        /**
         * 固定url抓取这一天第一页的数据
         */
        url = "https://epub.cnki.net/kns/brief/brief.aspx?pagename="+responseStr;
        responseStr = HttpConnect.HttpGet_OkHttp(url, headersMap);
        HttpConnect.checkHtml(responseStr, url);
        return responseStr;
    }


    public static String getNextUrl(String html, int curPage){
        Document htmlDoc = Jsoup.parse(html);
        Element lastTagA = htmlDoc.getElementsByClass("TitleLeftCell").first().getElementsByTag("a").last();
        if (!lastTagA.text().equals("下一页")) {
            return null;
        }

        String url = "https://epub.cnki.net/kns/brief/brief.aspx" + lastTagA.attr("href");
        int index = url.indexOf("&");
        url = url.substring(index);
        url = "https://epub.cnki.net/kns/brief/brief.aspx?curpage=" + curPage+url;
        return url;
    }


}
